import time
from datetime import datetime

import scrapy
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from spider_51job_company.items import Spider51JobCompanyItem


class Finace1Spider(scrapy.Spider):
    name = 'finace_1'
    allowed_domains = ['search.51job.com']

    # start_urls = ['http://search.51job.com/']

    def __init__(self, **kwargs):
        super().__init__(**kwargs)
        self.page_num = 1
        chrome_options = Options()
        self.driver = webdriver.Chrome(chrome_options=chrome_options)
        self.data = time.strftime('%Y-%m-%d')

    def start_requests(self):
        for i in range(4):  # 共有多少页
            self.page_num += i
            url = '''https://search.51job.com/list/000000,000000,0000,26,9,99,
            %25E5%25AE%25A4%25E5%2586%2585%25E8%25AE%25BE%25E8%25AE%25A1,2,{}.html?
            lang=c&postchannel=0000&workyear=99&cotype=05%252c11&degreefrom=99&jobterm=
            99&companysize=01&ord_field=0&dibiaoid=0&line=&welfare='''.format(int(self.page_num))
            print('spider当前{}页 start...'.format(self.page_num))
            yield scrapy.Request(url=url, meta={'page_num': self.page_num}, callback=self.parse)

    def parse(self, response):
        item = Spider51JobCompanyItem()
        page_num = response.meta['page_num']
        xpath = "//div[@class='j_joblist']/div[@class='e']"
        try:
            jobs = response.xpath(xpath)
            print('spider当前{}页，共{}条记录'.format(page_num, len(jobs)))
            for i in range(len(jobs)):
                company = response.xpath(xpath + "//div[@class='er']/a/text()").getall()[i]
                company_url = response.xpath(xpath + "//div[@class='er']/a/@href").getall()[i]
                item['company'] = company
                item['company_url'] = company_url
                yield item
        except Exception as e:
            print('第{}条spider error...{}'.format(page_num, e))
            f = open('spider_error_' + self.data + '.txt', 'a')
            f.write('\n {}spider error...: {}'.format(datetime.now(), e))
            f.write('\n {}\n'.format(response.url))
            f.close()

    def close(self, spider, reason):
        self.driver.close()
